import * as request from "request";
import * as fs from "fs";
import * as iconv from "iconv-lite";
import * as cheerio from "cheerio";

import { ReptileLogic } from "../app/logics/logic";
import { ReptileArticleEntity, ReptileArticleListEntity } from "../app/entitys/entity";

// setting
let time = 1000 * 60 * 60 * 12;  // 12/h
let url = 'http://www.cnzhixiang.com/news/103';

let lgc = new ReptileLogic();

main();

// 12/h爬取一次
setInterval(main, time);

function main() {
    console.log("爬取时间:" + FormatDateTime(new Date(), "yyyy-MM-dd hh:mm:ss"));

    request({
        method: "GET",
        url: url,
        encoding: null
    }, function (error, response, body) {

        if (error) {
            console.log(error);
        }
        else if (response && response.statusCode) {

            var ctype = response.headers['content-type'];

            if (ctype) {

                ctype = ctype.substr(ctype.indexOf('charset=') + 8, ctype.length);

            } else {

                ctype = "utf-8";

            }

            // console.log(body);
            // var buf = iconv.decode(body, ctype);

            eachNewsList(body);

        }

    });
}
function eachNewsList(str) {

    var re = /<li\s+class="catlist_li"><span\s+class="f_r px11 f_gray">(.*?)<\/span><a\s+href="(.*?)"\s+target="_blank"\s+title="(.*?)">(.*?)<\/a><\/li>/ig

    let arr;

    while ((arr = re.exec(str)) != null) {

        let ntt = new ReptileArticleListEntity();
        ntt.link = arr[2];
        ntt.title = arr[3];

        lgc.hasArticleData(ntt.link, function (res) {
            if (res.success) {
                if (res.total <= 0) {

                    eachNewsData(ntt);

                } else {
                    console.log("过滤数据:" + ntt.title);
                }
            } else {
                console.log("鉴权失败:" + ntt.title);
            }
        });
    }
}
function eachNewsData(ntt) {
    // 避免并发
    setTimeout(function () {
        request({
            method: "GET",
            url: ntt.link,
            encoding: null
        }, function (error, res, body) {

            if (error) {

                console.log(error);

            } else {

                var ctype = res.headers['content-type'];

                if (ctype) {

                    ctype = ctype.substr(ctype.indexOf('charset=') + 8, ctype.length);

                } else {

                    ctype = "utf-8";

                }

                // const $ = cheerio.load(iconv.decode(body, ctype)); gb2312 读取方式
                const $ = cheerio.load(body); // utf-8 读取方式

                const content = $('#article').html();

                const date = new Date();

                var time_f = FormatDateTime(new Date(), 'yyyy-MM-dd');

                if (ntt.title && ntt.title.trim()) {

                    lgc.addArticleData({ title: ntt.title, content: content, date: time_f, guid: ntt.link }, (res) => {

                        if (res.success) {
                            console.log("插入:" + ntt.title);
                        } else {
                            console.log(res.msg);
                        }
                    });
                } else {
                    console.log("异常过滤数据");
                }

            }
        })
    }, 1000);

}
function FormatDateTime(val, fmt) { //author: meizz 
    var date = new Date(val);

    var o = {
        "M+": date.getMonth() + 1, //月份 
        "d+": date.getDate(), //日 
        "h+": date.getHours(), //小时 
        "m+": date.getMinutes(), //分 
        "s+": date.getSeconds(), //秒 
        "q+": Math.floor((date.getMonth() + 3) / 3), //季度 
        "S": date.getMilliseconds() //毫秒 
    };
    if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (date.getFullYear() + "").substr(4 - RegExp.$1.length));
    for (var k in o)
        if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
    return fmt;
}